{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 导入基本包\n",
    "import os\n",
    "import re\n",
    "import jieba\n",
    "import numpy as py\n",
    "import pandas as pd\n",
    "import pyprind\n",
    "from bs4 import BeautifulSoup\n",
    "from nltk.corpus import stopwords\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn import metrics\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.naive_bayes import BernoulliNB\n",
    "from sklearn.preprocessing import LabelEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [##############################] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:44\n"
     ]
    }
   ],
   "source": [
    "# 加载数据集\n",
    "# 数据加载\n",
    "path = 'data/aclImdb'\n",
    "labels = {'pos': 1, 'neg': 0}\n",
    "# 设置进度条\n",
    "pbar = pyprind.ProgBar(25000)\n",
    "\n",
    "\n",
    "def getData(type):\n",
    "    \"\"\"\n",
    "    数据加载，将原数据顺序打乱\n",
    "    :param: 需要加载的数据集类型train、test\n",
    "    :return: DataFrame\n",
    "    \"\"\"\n",
    "    data = pd.DataFrame()\n",
    "    for j in ('pos', 'neg'):\n",
    "        # 拼接路径\n",
    "        type_path = os.path.join(path, type, j)\n",
    "        for k in os.listdir(type_path):\n",
    "            # 以文件流的形式打开文件\n",
    "            with open(os.path.join(type_path, k), 'r', encoding='utf-8') as file:\n",
    "                comment = file.read()\n",
    "            data = data.append([[comment, labels[j]]], ignore_index=True)\n",
    "            pbar.update()\n",
    "    data.columns = ['comment', 'sentiment']\n",
    "    # 原数据规律性太强，将原数据顺序打乱\n",
    "    resultData = data.sample(frac=1).reset_index(drop=True)\n",
    "    return resultData\n",
    "\n",
    "\n",
    "train_data = getData(\"train\")\n",
    "test_data = getData(\"test\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "data": {
      "text/plain": "                                             comment  sentiment\n0  A very interesting documentary - certainly a l...          1\n1  This movie sets out to do something very parti...          1\n2  \"Cleo's Second Husband\" is an amateurish attem...          0\n3  \"Caligula\" shares many of the same attributes ...          0\n4  I bought this DVD from Walmart for cheap, thin...          1",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>comment</th>\n      <th>sentiment</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>A very interesting documentary - certainly a l...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>This movie sets out to do something very parti...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>\"Cleo's Second Husband\" is an amateurish attem...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>\"Caligula\" shares many of the same attributes ...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>I bought this DVD from Walmart for cheap, thin...</td>\n      <td>1</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "data": {
      "text/plain": "                                             comment  sentiment\n0  (A possible spoiler or two) <br /><br /> \"Soul...          0\n1  Horrible acting, Bad story line, cheesy makeup...          0\n2  Trawling through the Sci Fi weeklies section o...          1\n3  O my gosh... Just give me a minute to breath. ...          1\n4  I saw \"Fever Pitch\" sort of by accident; it wa...          1",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>comment</th>\n      <th>sentiment</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>(A possible spoiler or two) &lt;br /&gt;&lt;br /&gt; \"Soul...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>Horrible acting, Bad story line, cheesy makeup...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>Trawling through the Sci Fi weeklies section o...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>O my gosh... Just give me a minute to breath. ...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>I saw \"Fever Pitch\" sort of by accident; it wa...</td>\n      <td>1</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "outputs": [],
   "source": [
    "train_comment=train_data[\"comment\"]\n",
    "test_comment=test_data[\"comment\"]\n",
    "train_label=train_data[\"sentiment\"]\n",
    "test_label=test_data[\"sentiment\"]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25000\n"
     ]
    }
   ],
   "source": [
    "def cut_word(text):\n",
    "    return ' '.join(jieba.cut(text))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "def html_to_text(sentence):\n",
    "    \"\"\"\n",
    "    因为评论来源于爬虫抓取，因此可能会携带Html标签，\n",
    "    本函数的作用是使用bs4库去除文本中的HTML标签及非法字符\n",
    "    :param sentence: str\n",
    "    :return: str\n",
    "    \"\"\"\n",
    "    # 把大写转化为小写\n",
    "    sentence = sentence.lower()\n",
    "    # sentence = re.sub(\"<br />\", \" \", sentence)\n",
    "\n",
    "    # 去除Html标签\n",
    "    soup = BeautifulSoup(sentence, 'lxml')\n",
    "    sentence = soup.get_text()\n",
    "\n",
    "    return sentence\n",
    "\n",
    "\n",
    "def tokenizer(sentence):\n",
    "    \"\"\"\n",
    "    分词操作，由于是英文，直接用空格做切分即可，对于中文分词用结巴分词库来操作\n",
    "    :param sentence: str\n",
    "    :return: [word,word,word]\n",
    "    \"\"\"\n",
    "    # 定义正则过滤器\n",
    "    filters = ['!', '\"', '#', '$', '%', '&', '\\(', '\\)', '\\*', '\\+', ',', '-', '\\.', '/', ':', ';', '<', '=', '>',\n",
    "               '\\?', '@', '\\[', '\\\\', '\\]', '^', '_', '`', '\\{', '\\|', '\\}', '~', '\\t', '\\n', '\\x97', '\\x96', '”',\n",
    "               '“', ]\n",
    "    sentence = re.sub(\"|\".join(filters), \" \", sentence)\n",
    "\n",
    "    result = [i for i in sentence.split(\" \") if len(i) > 0]\n",
    "    return result\n",
    "\n",
    "\n",
    "def original_word(sentence_list):\n",
    "    \"\"\"\n",
    "    使用PorterStemmer还原词干\n",
    "    :param sentence_list: [running,word,words]\n",
    "    :return: [running,word,word]\n",
    "    \"\"\"\n",
    "    # 还原词根\n",
    "    word_list = []\n",
    "    porter = PorterStemmer()\n",
    "    for word in sentence_list:\n",
    "        sentence = porter.stem(word)\n",
    "        word_list.append(sentence)\n",
    "    return word_list\n",
    "\n",
    "\n",
    "def remove_stopwords(sentence_list):\n",
    "    \"\"\"\n",
    "    去除停用词\n",
    "    :param sentence_list: [str,str,str]\n",
    "    :return: [str,str,str]\n",
    "    \"\"\"\n",
    "    filtered_words = []\n",
    "    # 英文停用词\n",
    "    stop_words = stopwords.words('english')\n",
    "    for filters in ['!', ',', '.', '?', '-s', '-ly', '</s>', 's', '1', '2', '3', '4', '5', '6', '7', '8', '9']:\n",
    "        stop_words.append(filters)\n",
    "    for word in sentence_list:\n",
    "        if word not in stop_words:\n",
    "            filtered_words.append(word)\n",
    "    return filtered_words\n",
    "\n",
    "\n",
    "def get_sentence(text):\n",
    "    \"\"\"\n",
    "     获取处理好的数据\n",
    "    :param text: [str,str,str]\n",
    "    :return: [str,str,str]\n",
    "    \"\"\"\n",
    "\n",
    "    # 文本清洗\n",
    "    sentence = html_to_text(text)\n",
    "    # 文本分词\n",
    "    tokenizer_list = tokenizer(sentence)\n",
    "    # 去除停用词\n",
    "    words_list = remove_stopwords(tokenizer_list)\n",
    "    # 还原词干\n",
    "    #finall_word_list=pro.original_word(words_list)\n",
    "\n",
    "    return words_list"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}